# coding:utf-8

from scrapy.contrib.spiders import CrawlSpider
from ..items import KoojuItem
import scrapy
import bs4


class KoojuSpider(CrawlSpider):
    #
    # 获取 酷居网 规则
    #

    name = 'koojuspider'
    allowed_domains = ['kooju.cn']
    start_urls = ['http://kooju.cn/house/']

    def parse(self, response):
        data = response.body
        soup = bs4.BeautifulSoup(data, 'lxml')

        #
        # 获取所有城市
        #

        citybox = soup.select('.select_li > ul')
        for child in citybox:
            cols = child.li.find_all('a')
            for col in cols:
                city = col.get_text()
                if city != '不限':
                    city_url = col.get('href')
                    city_id = col.get('href').replace('http://', '').replace('.kooju.cn/house/', '')
                    citys = {
                        'website': '酷居网', 'web_url': 'kooju.cn',
                        'city': city, 'city_id': city_id,
                    }
                    yield scrapy.Request(city_url, callback=self.parse_cityPage_url, meta=citys)

    def parse_cityPage_url(self, response):
        meta = response.meta
        data = response.body
        city_id = meta['city_id']
        soup = bs4.BeautifulSoup(data, 'lxml')

        #
        # 进行 城市翻页
        #

        pages = soup.select('.page > a ')[-2]
        page_num = int(pages.get_text().strip())
        for n in range(page_num):
            site_url = 'http://' + city_id + '.kooju.cn/house/' + str(n + 1) + '.html'
            yield scrapy.Request(site_url, callback=self.parse_city_estate, meta=meta)

    def parse_city_estate(self, response):
        meta = response.meta
        data = response.body
        soup = bs4.BeautifulSoup(data, 'lxml')

        #
        # 获取楼盘信息
        #

        estates = soup.select('.house_txt_bt > a ')
        for child in estates:
            estate = child.get_text()
            estate_url = child.get('href')
            estate_id = child.get('href').replace('http://www.kooju.cn/house/', '').replace('/', '')
            if estate != '':
                item = KoojuItem()
                item['website'] = meta['website']
                item['web_url'] = meta['web_url']
                item['city'] = meta['city']
                item['area'] = ''
                item['city_id'] = meta['city_id']
                item['estate'] = estate
                item['estate_id'] = estate_id
                item['estate_url'] = estate_url
                yield item
